{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a supplementary file for the tutorial this week and can be safely ignored\n",
    "solutions will also be posted for this at the end of the week if you are interested\n",
    "if you find any errors please notify me at omarghattas1991@gmail.com"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Aim: We are going to explore three fundamental algorithms that are the basis for deep learning:\n",
    "1. the Perceptron training rule\n",
    "2. Gradient Descent\n",
    "3. Stochastic Gradient Descent\n",
    "\n",
    "At the end of this lab you should be comfortable with perceptron training and the basics of \n",
    "gradient descent -> be able to start thinking about neural networks and building them from scratch \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "np.random.seed(43)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_scatter(X, y, w):\n",
    "    pos_points = X[np.where(y==1)[0]]\n",
    "    neg_points = X[np.where(y==-1)[0]]\n",
    "    plt.scatter(pos_points[:, 1], pos_points[:, 2], color='blue')\n",
    "    plt.scatter(neg_points[:, 1], neg_points[:, 2], color='red')\n",
    "    xx = np.linspace(-6,6)\n",
    "    yy = -w[0]/w[2] - w[1]/w[2] * xx\n",
    "    plt.plot(xx, yy, color='orange')\n",
    "    \n",
    "    ratio = (w[2]/w[1] + w[1]/w[2])\n",
    "    xpt = (-1*w[0] / w[2]) * 1/ratio\n",
    "    ypt = (-1*w[0] / w[1]) * 1/ratio\n",
    "    \n",
    "    plt.axes().arrow(xpt, ypt, w[1], w[2], head_width=0.2, color='orange')\n",
    "    plt.axis('equal')\n",
    "    plt.show()\n",
    "    \n",
    "#plot_scatter(X, y, w=np.array([0,-1,3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data(n=20):\n",
    "    dist_01 = np.random.multivariate_normal(np.array([2,2]), np.array([[1,0.9],[0.5,1]]), n)\n",
    "    dist_02 = np.random.multivariate_normal(np.array([-2,-2]),np.array([[1,0],[0.3,1]]), n)\n",
    "    X = np.concatenate((np.ones(2*n).reshape(-1,1), np.concatenate((dist_01, dist_02))), axis=1)\n",
    "    y = np.concatenate((np.ones(n), -1*np.ones(n)))\n",
    "    shuffle_idx = np.random.choice(2*n, size=2*n, replace=False)\n",
    "    X = X[shuffle_idx]\n",
    "    y = y[shuffle_idx]\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perceptron Training Rule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def train_perceptron(X, y, eta=1):\n",
    "    w = np.array([-2,-1,3])\n",
    "    nmb_data = X.shape[0]\n",
    "    '''\n",
    "    your code here\n",
    "    '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X, y = generate_data()\n",
    "train_perceptron(X, y, eta=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x,a=1):\n",
    "    # your code here\n",
    "\n",
    "def grad_sigmoid(x, a=1):\n",
    "    # your code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot sig functions \n",
    "xx = np.linspace(-6,6,1000)\n",
    "a_vals = [0.4,1,2,3]\n",
    "cols = ['red', 'blue', 'orange', 'green']\n",
    "y_sig = [sigmoid(xx, a_val) for a_val in a_vals]\n",
    "y_grad_sig = [grad_sigmoid(xx, a_val) for a_val in a_vals]\n",
    "\n",
    "for i in range(len(a_vals)):\n",
    "    label = str(a_vals[i])\n",
    "    plt.plot(xx, y_sig[i], label=rf'$a={label}$', color=cols[i])\n",
    "    plt.plot(xx, y_grad_sig[i], color=cols[i])\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_i(w, x_i, y_i):\n",
    "    '''squared loss for i-th data point'''\n",
    "    # your code here\n",
    "    \n",
    "def grad_loss_i(w, x_i, y_i):\n",
    "    '''grad loss for i-th data point'''\n",
    "    # your code here\n",
    "\n",
    "def gradient_descent(X, y, eta, T):\n",
    "    nmb_data = X.shape[0]\n",
    "    w = np.array([-2,-1,-3])\n",
    "    plot_scatter(X, y, w)\n",
    "    # your code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X, y = generate_data(50)\n",
    "gradient_descent(X, y, 0.2, 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stochastic Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stochastic_gradient_descent(X, y, eta, T):\n",
    "    # your code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# generate data\n",
    "X, y = generate_data()\n",
    "stochastic_gradient_descent(X, y, 0.5, 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
